{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 通过ssh 下载远程服务器上的文件"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-03-28T08:23:33.276176Z",
     "start_time": "2024-03-28T08:21:48.554251Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sucess fetch remote dir info,use 4.33 s\n",
      "there are 90 files to loacal \n",
      "start download ==============>\n",
      "AC_type2_2024-03-20.parquet size 7.25 MB ,use 14.85 s\n",
      "AC_type1_2024-03-20.parquet size 1.93 MB ,use 5.84 s\n",
      "AM_type2_2024-03-20.parquet size 19.61 MB ,use 19.89 s\n",
      "AM_type1_2024-03-20.parquet size 3.26 MB ,use 5.54 s\n",
      "AM_type3_2024-03-20.parquet size 0.25 MB ,use 4.14 s\n",
      "AL_type2_2024-03-20.parquet size 0.59 MB ,use 3.12 s\n",
      "AL_type1_2024-03-20.parquet size 0.5 MB ,use 3.39 s\n",
      "AG_type2_2024-03-20.parquet size 1.17 MB ,use 6.23 s\n",
      "AG_type1_2024-03-20.parquet size 0.38 MB ,use 3.11 s\n",
      "AEC_type2_2024-03-20.parquet size 1.04 MB ,use 6.44 s\n",
      "AM_type2_2024-03-21.parquet size 20.19 MB ,use 20.81 s\n"
     ]
    },
    {
     "ename": "SFTPError",
     "evalue": "Garbage packet received",
     "output_type": "error",
     "traceback": [
      "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[0;31mKeyboardInterrupt\u001B[0m                         Traceback (most recent call last)",
      "File \u001B[0;32m~/anaconda3/envs/py311/lib/python3.11/site-packages/paramiko/sftp_client.py:799\u001B[0m, in \u001B[0;36mSFTPClient.getfo\u001B[0;34m(self, remotepath, fl, callback, prefetch, max_concurrent_prefetch_requests)\u001B[0m\n\u001B[1;32m    798\u001B[0m     fr\u001B[38;5;241m.\u001B[39mprefetch(file_size, max_concurrent_prefetch_requests)\n\u001B[0;32m--> 799\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_transfer_with_callback\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m    800\u001B[0m \u001B[43m    \u001B[49m\u001B[43mreader\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mfr\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mwriter\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mfl\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mfile_size\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mfile_size\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mcallback\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mcallback\u001B[49m\n\u001B[1;32m    801\u001B[0m \u001B[43m\u001B[49m\u001B[43m)\u001B[49m\n",
      "File \u001B[0;32m~/anaconda3/envs/py311/lib/python3.11/site-packages/paramiko/sftp_client.py:678\u001B[0m, in \u001B[0;36mSFTPClient._transfer_with_callback\u001B[0;34m(self, reader, writer, file_size, callback)\u001B[0m\n\u001B[1;32m    677\u001B[0m \u001B[38;5;28;01mwhile\u001B[39;00m \u001B[38;5;28;01mTrue\u001B[39;00m:\n\u001B[0;32m--> 678\u001B[0m     data \u001B[38;5;241m=\u001B[39m \u001B[43mreader\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mread\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;241;43m32768\u001B[39;49m\u001B[43m)\u001B[49m\n\u001B[1;32m    679\u001B[0m     writer\u001B[38;5;241m.\u001B[39mwrite(data)\n",
      "File \u001B[0;32m~/anaconda3/envs/py311/lib/python3.11/site-packages/paramiko/file.py:203\u001B[0m, in \u001B[0;36mBufferedFile.read\u001B[0;34m(self, size)\u001B[0m\n\u001B[1;32m    202\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m--> 203\u001B[0m     new_data \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_read\u001B[49m\u001B[43m(\u001B[49m\u001B[43mread_size\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    204\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m \u001B[38;5;167;01mEOFError\u001B[39;00m:\n",
      "File \u001B[0;32m~/anaconda3/envs/py311/lib/python3.11/site-packages/paramiko/sftp_file.py:182\u001B[0m, in \u001B[0;36mSFTPFile._read\u001B[0;34m(self, size)\u001B[0m\n\u001B[1;32m    181\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_prefetching:\n\u001B[0;32m--> 182\u001B[0m     data \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_read_prefetch\u001B[49m\u001B[43m(\u001B[49m\u001B[43msize\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    183\u001B[0m     \u001B[38;5;28;01mif\u001B[39;00m data \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n",
      "File \u001B[0;32m~/anaconda3/envs/py311/lib/python3.11/site-packages/paramiko/sftp_file.py:162\u001B[0m, in \u001B[0;36mSFTPFile._read_prefetch\u001B[0;34m(self, size)\u001B[0m\n\u001B[1;32m    161\u001B[0m     \u001B[38;5;28;01mbreak\u001B[39;00m\n\u001B[0;32m--> 162\u001B[0m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43msftp\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_read_response\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    163\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_check_exception()\n",
      "File \u001B[0;32m~/anaconda3/envs/py311/lib/python3.11/site-packages/paramiko/sftp_client.py:887\u001B[0m, in \u001B[0;36mSFTPClient._read_response\u001B[0;34m(self, waitfor)\u001B[0m\n\u001B[1;32m    886\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m--> 887\u001B[0m     t, data \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_read_packet\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    888\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m \u001B[38;5;167;01mEOFError\u001B[39;00m \u001B[38;5;28;01mas\u001B[39;00m e:\n",
      "File \u001B[0;32m~/anaconda3/envs/py311/lib/python3.11/site-packages/paramiko/sftp.py:218\u001B[0m, in \u001B[0;36mBaseSFTP._read_packet\u001B[0;34m(self)\u001B[0m\n\u001B[1;32m    217\u001B[0m size \u001B[38;5;241m=\u001B[39m struct\u001B[38;5;241m.\u001B[39munpack(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124m>I\u001B[39m\u001B[38;5;124m\"\u001B[39m, x)[\u001B[38;5;241m0\u001B[39m]\n\u001B[0;32m--> 218\u001B[0m data \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_read_all\u001B[49m\u001B[43m(\u001B[49m\u001B[43msize\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    219\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39multra_debug:\n",
      "File \u001B[0;32m~/anaconda3/envs/py311/lib/python3.11/site-packages/paramiko/sftp.py:196\u001B[0m, in \u001B[0;36mBaseSFTP._read_all\u001B[0;34m(self, n)\u001B[0m\n\u001B[1;32m    195\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[0;32m--> 196\u001B[0m     x \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43msock\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mrecv\u001B[49m\u001B[43m(\u001B[49m\u001B[43mn\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    198\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mlen\u001B[39m(x) \u001B[38;5;241m==\u001B[39m \u001B[38;5;241m0\u001B[39m:\n",
      "File \u001B[0;32m~/anaconda3/envs/py311/lib/python3.11/site-packages/paramiko/channel.py:697\u001B[0m, in \u001B[0;36mChannel.recv\u001B[0;34m(self, nbytes)\u001B[0m\n\u001B[1;32m    696\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m--> 697\u001B[0m     out \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43min_buffer\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mread\u001B[49m\u001B[43m(\u001B[49m\u001B[43mnbytes\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mtimeout\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    698\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m PipeTimeout:\n",
      "File \u001B[0;32m~/anaconda3/envs/py311/lib/python3.11/site-packages/paramiko/buffered_pipe.py:150\u001B[0m, in \u001B[0;36mBufferedPipe.read\u001B[0;34m(self, nbytes, timeout)\u001B[0m\n\u001B[1;32m    149\u001B[0m then \u001B[38;5;241m=\u001B[39m time\u001B[38;5;241m.\u001B[39mtime()\n\u001B[0;32m--> 150\u001B[0m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_cv\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mwait\u001B[49m\u001B[43m(\u001B[49m\u001B[43mtimeout\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    151\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m timeout \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n",
      "File \u001B[0;32m~/anaconda3/envs/py311/lib/python3.11/threading.py:320\u001B[0m, in \u001B[0;36mCondition.wait\u001B[0;34m(self, timeout)\u001B[0m\n\u001B[1;32m    319\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m timeout \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n\u001B[0;32m--> 320\u001B[0m     \u001B[43mwaiter\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43macquire\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    321\u001B[0m     gotit \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;01mTrue\u001B[39;00m\n",
      "\u001B[0;31mKeyboardInterrupt\u001B[0m: ",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001B[0;31mSFTPError\u001B[0m                                 Traceback (most recent call last)",
      "Cell \u001B[0;32mIn[1], line 57\u001B[0m\n\u001B[1;32m     55\u001B[0m start \u001B[38;5;241m=\u001B[39m time\u001B[38;5;241m.\u001B[39mtime()\n\u001B[1;32m     56\u001B[0m file_size \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mround\u001B[39m(file_attr\u001B[38;5;241m.\u001B[39mst_size\u001B[38;5;241m/\u001B[39m\u001B[38;5;241m1048576\u001B[39m,\u001B[38;5;241m2\u001B[39m)\n\u001B[0;32m---> 57\u001B[0m \u001B[43msftp\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mget\u001B[49m\u001B[43m(\u001B[49m\u001B[43mremote_file_path\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mlocal_file_path\u001B[49m\u001B[43m)\u001B[49m  \n\u001B[1;32m     58\u001B[0m end \u001B[38;5;241m=\u001B[39m time\u001B[38;5;241m.\u001B[39mtime()\n\u001B[1;32m     59\u001B[0m \u001B[38;5;28mprint\u001B[39m(\u001B[38;5;124mf\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;132;01m{\u001B[39;00mfile_name\u001B[38;5;132;01m}\u001B[39;00m\u001B[38;5;124m size \u001B[39m\u001B[38;5;132;01m{\u001B[39;00mfile_size\u001B[38;5;132;01m}\u001B[39;00m\u001B[38;5;124m MB ,use \u001B[39m\u001B[38;5;132;01m{\u001B[39;00m\u001B[38;5;28mround\u001B[39m(end\u001B[38;5;250m \u001B[39m\u001B[38;5;241m-\u001B[39mstart,\u001B[38;5;241m2\u001B[39m)\u001B[38;5;132;01m}\u001B[39;00m\u001B[38;5;124m s\u001B[39m\u001B[38;5;124m\"\u001B[39m) \n",
      "File \u001B[0;32m~/anaconda3/envs/py311/lib/python3.11/site-packages/paramiko/sftp_client.py:840\u001B[0m, in \u001B[0;36mSFTPClient.get\u001B[0;34m(self, remotepath, localpath, callback, prefetch, max_concurrent_prefetch_requests)\u001B[0m\n\u001B[1;32m    811\u001B[0m \u001B[38;5;250m\u001B[39m\u001B[38;5;124;03m\"\"\"\u001B[39;00m\n\u001B[1;32m    812\u001B[0m \u001B[38;5;124;03mCopy a remote file (``remotepath``) from the SFTP server to the local\u001B[39;00m\n\u001B[1;32m    813\u001B[0m \u001B[38;5;124;03mhost as ``localpath``.  Any exception raised by operations will be\u001B[39;00m\n\u001B[0;32m   (...)\u001B[0m\n\u001B[1;32m    837\u001B[0m \u001B[38;5;124;03m    Added ``max_concurrent_prefetch_requests``.\u001B[39;00m\n\u001B[1;32m    838\u001B[0m \u001B[38;5;124;03m\"\"\"\u001B[39;00m\n\u001B[1;32m    839\u001B[0m \u001B[38;5;28;01mwith\u001B[39;00m \u001B[38;5;28mopen\u001B[39m(localpath, \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mwb\u001B[39m\u001B[38;5;124m\"\u001B[39m) \u001B[38;5;28;01mas\u001B[39;00m fl:\n\u001B[0;32m--> 840\u001B[0m     size \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mgetfo\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m    841\u001B[0m \u001B[43m        \u001B[49m\u001B[43mremotepath\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m    842\u001B[0m \u001B[43m        \u001B[49m\u001B[43mfl\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m    843\u001B[0m \u001B[43m        \u001B[49m\u001B[43mcallback\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m    844\u001B[0m \u001B[43m        \u001B[49m\u001B[43mprefetch\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m    845\u001B[0m \u001B[43m        \u001B[49m\u001B[43mmax_concurrent_prefetch_requests\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m    846\u001B[0m \u001B[43m    \u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    847\u001B[0m s \u001B[38;5;241m=\u001B[39m os\u001B[38;5;241m.\u001B[39mstat(localpath)\n\u001B[1;32m    848\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m s\u001B[38;5;241m.\u001B[39mst_size \u001B[38;5;241m!=\u001B[39m size:\n",
      "File \u001B[0;32m~/anaconda3/envs/py311/lib/python3.11/site-packages/paramiko/sftp_client.py:796\u001B[0m, in \u001B[0;36mSFTPClient.getfo\u001B[0;34m(self, remotepath, fl, callback, prefetch, max_concurrent_prefetch_requests)\u001B[0m\n\u001B[1;32m    769\u001B[0m \u001B[38;5;250m\u001B[39m\u001B[38;5;124;03m\"\"\"\u001B[39;00m\n\u001B[1;32m    770\u001B[0m \u001B[38;5;124;03mCopy a remote file (``remotepath``) from the SFTP server and write to\u001B[39;00m\n\u001B[1;32m    771\u001B[0m \u001B[38;5;124;03man open file or file-like object, ``fl``.  Any exception raised by\u001B[39;00m\n\u001B[0;32m   (...)\u001B[0m\n\u001B[1;32m    793\u001B[0m \u001B[38;5;124;03m    Added ``max_concurrent_prefetch_requests``.\u001B[39;00m\n\u001B[1;32m    794\u001B[0m \u001B[38;5;124;03m\"\"\"\u001B[39;00m\n\u001B[1;32m    795\u001B[0m file_size \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mstat(remotepath)\u001B[38;5;241m.\u001B[39mst_size\n\u001B[0;32m--> 796\u001B[0m \u001B[43m\u001B[49m\u001B[38;5;28;43;01mwith\u001B[39;49;00m\u001B[43m \u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mopen\u001B[49m\u001B[43m(\u001B[49m\u001B[43mremotepath\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43mrb\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[43m)\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;28;43;01mas\u001B[39;49;00m\u001B[43m \u001B[49m\u001B[43mfr\u001B[49m\u001B[43m:\u001B[49m\n\u001B[1;32m    797\u001B[0m \u001B[43m    \u001B[49m\u001B[38;5;28;43;01mif\u001B[39;49;00m\u001B[43m \u001B[49m\u001B[43mprefetch\u001B[49m\u001B[43m:\u001B[49m\n\u001B[1;32m    798\u001B[0m \u001B[43m        \u001B[49m\u001B[43mfr\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mprefetch\u001B[49m\u001B[43m(\u001B[49m\u001B[43mfile_size\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mmax_concurrent_prefetch_requests\u001B[49m\u001B[43m)\u001B[49m\n",
      "File \u001B[0;32m~/anaconda3/envs/py311/lib/python3.11/site-packages/paramiko/util.py:290\u001B[0m, in \u001B[0;36mClosingContextManager.__exit__\u001B[0;34m(self, type, value, traceback)\u001B[0m\n\u001B[1;32m    289\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21m__exit__\u001B[39m(\u001B[38;5;28mself\u001B[39m, \u001B[38;5;28mtype\u001B[39m, value, traceback):\n\u001B[0;32m--> 290\u001B[0m     \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mclose\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\n",
      "File \u001B[0;32m~/anaconda3/envs/py311/lib/python3.11/site-packages/paramiko/sftp_file.py:82\u001B[0m, in \u001B[0;36mSFTPFile.close\u001B[0;34m(self)\u001B[0m\n\u001B[1;32m     78\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mclose\u001B[39m(\u001B[38;5;28mself\u001B[39m):\n\u001B[1;32m     79\u001B[0m \u001B[38;5;250m    \u001B[39m\u001B[38;5;124;03m\"\"\"\u001B[39;00m\n\u001B[1;32m     80\u001B[0m \u001B[38;5;124;03m    Close the file.\u001B[39;00m\n\u001B[1;32m     81\u001B[0m \u001B[38;5;124;03m    \"\"\"\u001B[39;00m\n\u001B[0;32m---> 82\u001B[0m     \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_close\u001B[49m\u001B[43m(\u001B[49m\u001B[43masync_\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43;01mFalse\u001B[39;49;00m\u001B[43m)\u001B[49m\n",
      "File \u001B[0;32m~/anaconda3/envs/py311/lib/python3.11/site-packages/paramiko/sftp_file.py:104\u001B[0m, in \u001B[0;36mSFTPFile._close\u001B[0;34m(self, async_)\u001B[0m\n\u001B[1;32m    102\u001B[0m         \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39msftp\u001B[38;5;241m.\u001B[39m_async_request(\u001B[38;5;28mtype\u001B[39m(\u001B[38;5;28;01mNone\u001B[39;00m), CMD_CLOSE, \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mhandle)\n\u001B[1;32m    103\u001B[0m     \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[0;32m--> 104\u001B[0m         \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43msftp\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_request\u001B[49m\u001B[43m(\u001B[49m\u001B[43mCMD_CLOSE\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mhandle\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    105\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m \u001B[38;5;167;01mEOFError\u001B[39;00m:\n\u001B[1;32m    106\u001B[0m     \u001B[38;5;66;03m# may have outlived the Transport connection\u001B[39;00m\n\u001B[1;32m    107\u001B[0m     \u001B[38;5;28;01mpass\u001B[39;00m\n",
      "File \u001B[0;32m~/anaconda3/envs/py311/lib/python3.11/site-packages/paramiko/sftp_client.py:857\u001B[0m, in \u001B[0;36mSFTPClient._request\u001B[0;34m(self, t, *args)\u001B[0m\n\u001B[1;32m    855\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21m_request\u001B[39m(\u001B[38;5;28mself\u001B[39m, t, \u001B[38;5;241m*\u001B[39margs):\n\u001B[1;32m    856\u001B[0m     num \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_async_request(\u001B[38;5;28mtype\u001B[39m(\u001B[38;5;28;01mNone\u001B[39;00m), t, \u001B[38;5;241m*\u001B[39margs)\n\u001B[0;32m--> 857\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_read_response\u001B[49m\u001B[43m(\u001B[49m\u001B[43mnum\u001B[49m\u001B[43m)\u001B[49m\n",
      "File \u001B[0;32m~/anaconda3/envs/py311/lib/python3.11/site-packages/paramiko/sftp_client.py:887\u001B[0m, in \u001B[0;36mSFTPClient._read_response\u001B[0;34m(self, waitfor)\u001B[0m\n\u001B[1;32m    885\u001B[0m \u001B[38;5;28;01mwhile\u001B[39;00m \u001B[38;5;28;01mTrue\u001B[39;00m:\n\u001B[1;32m    886\u001B[0m     \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m--> 887\u001B[0m         t, data \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_read_packet\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    888\u001B[0m     \u001B[38;5;28;01mexcept\u001B[39;00m \u001B[38;5;167;01mEOFError\u001B[39;00m \u001B[38;5;28;01mas\u001B[39;00m e:\n\u001B[1;32m    889\u001B[0m         \u001B[38;5;28;01mraise\u001B[39;00m SSHException(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mServer connection dropped: \u001B[39m\u001B[38;5;132;01m{}\u001B[39;00m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;241m.\u001B[39mformat(e))\n",
      "File \u001B[0;32m~/anaconda3/envs/py311/lib/python3.11/site-packages/paramiko/sftp.py:216\u001B[0m, in \u001B[0;36mBaseSFTP._read_packet\u001B[0;34m(self)\u001B[0m\n\u001B[1;32m    213\u001B[0m \u001B[38;5;66;03m# most sftp servers won't accept packets larger than about 32k, so\u001B[39;00m\n\u001B[1;32m    214\u001B[0m \u001B[38;5;66;03m# anything with the high byte set (> 16MB) is just garbage.\u001B[39;00m\n\u001B[1;32m    215\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m byte_ord(x[\u001B[38;5;241m0\u001B[39m]):\n\u001B[0;32m--> 216\u001B[0m     \u001B[38;5;28;01mraise\u001B[39;00m SFTPError(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mGarbage packet received\u001B[39m\u001B[38;5;124m\"\u001B[39m)\n\u001B[1;32m    217\u001B[0m size \u001B[38;5;241m=\u001B[39m struct\u001B[38;5;241m.\u001B[39munpack(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124m>I\u001B[39m\u001B[38;5;124m\"\u001B[39m, x)[\u001B[38;5;241m0\u001B[39m]\n\u001B[1;32m    218\u001B[0m data \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_read_all(size)\n",
      "\u001B[0;31mSFTPError\u001B[0m: Garbage packet received"
     ]
    }
   ],
   "source": [
    "import paramiko  \n",
    "import os  \n",
    "import re\n",
    "import time\n",
    "from concurrent.futures import ThreadPoolExecutor, as_completed\n",
    "import traceback\n",
    "from tqdm import tqdm\n",
    "\n",
    "\n",
    "hostname = '47.253.62.54'  \n",
    "port = 9988  \n",
    "username = 'root'  \n",
    "password = 'Ikeyvdrq14B+1olzUZpKGma'  \n",
    "\n",
    "remote_dir = '/data/apps/data/risk/jupyterdir/data/urule_log'  \n",
    "local_dir = '/home/loong/jupyter/YuChuang/urule_log/data' \n",
    "# start_date='2023-10-23'\n",
    "# start_date='2024-01-15'\n",
    "start_date='2024-03-20'\n",
    "\n",
    "# 创建SSH客户端  \n",
    "client = paramiko.SSHClient()  \n",
    "# 自动添加服务器主机名及主机密钥到本地HostKeys对象，并保存  \n",
    "client.set_missing_host_key_policy(paramiko.AutoAddPolicy())  \n",
    "# 连接SSH服务端，以用户名和密码进行认证  \n",
    "client.connect(hostname, port=port, username=username, password=password)\n",
    "# 使用SFTP传输文件  \n",
    "sftp = client.open_sftp()  \n",
    "\n",
    "# 列出远程目录中的文件  \n",
    "start = time.time()\n",
    "remote_attrs = sftp.listdir_attr(remote_dir)  \n",
    "end = time.time()\n",
    "print(f\"sucess fetch remote dir info,use {round(end-start,2)} s\")\n",
    "\n",
    "file_attr_2locals=[]\n",
    "for file_attr in remote_attrs:\n",
    "    file_name = file_attr.filename\n",
    "    if ( any( file_type in file_name for file_type in ['.parquet','.pqt','.pickle','.pkl','.csv','.xlsx'] ) ):\n",
    "        pattern = r\"\\d{4}-\\d{2}-\\d{2}\" \n",
    "        match = re.search(pattern, file_name)\n",
    "        date = match.group()\n",
    "        if date>= start_date:\n",
    "            file_attr_2locals.append(file_attr)\n",
    "\n",
    "print(f\"there are {len(file_attr_2locals)} files to loacal \")\n",
    "\n",
    "print(f\"start download ==============>\")\n",
    "\n",
    "start1 = time.time()\n",
    "for file_attr in file_attr_2locals:\n",
    "    file_name = file_attr.filename\n",
    "    remote_file_path = os.path.join(remote_dir,file_name)\n",
    "    local_file_path = os.path.join(local_dir,file_name)\n",
    "    start = time.time()\n",
    "    file_size = round(file_attr.st_size/1048576,2)\n",
    "    sftp.get(remote_file_path, local_file_path)  \n",
    "    end = time.time()\n",
    "    print(f\"{file_name} size {file_size} MB ,use {round(end -start,2)} s\") \n",
    "end1 = time.time()\n",
    "\n",
    "print(f\"数据下载完成，共下载{len(file_attr_2locals)}个文件，总共耗时 {round(end1-start1,2)} s\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "sftp.close()\n",
    "client.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "py311",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
